package com.az.alibaba.downloader;

import hirondelle.date4j.DateTime;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.sql.Connection;
import java.sql.SQLException;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.TimeZone;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.handlers.MapHandler;
import org.apache.commons.dbutils.handlers.MapListHandler;
import org.apache.commons.dbutils.handlers.ScalarHandler;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.StatusLine;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.utils.URLEncodedUtils;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.az.alibaba.common.db.Db;

public class DemoSpider {
	private static final Logger log = LoggerFactory.getLogger(DemoSpider.class);
	private static Random random = new Random(System.currentTimeMillis());

	private static int redirectNum = 0;
	private static int maxRedirect = 20;
	private static boolean first = true;

	private static RequestConfig config = RequestConfig.custom().setConnectTimeout(5000).setSocketTimeout(5000).setMaxRedirects(1).setRedirectsEnabled(false)
			.build();

	private static String queryUnuseWord = "select * from keyword as keyword order by use_tm limit 1";
	private static String updateUseWord = "update keyword set use_tm = now() where id = ?";
	private static String insertUrl = "insert into record (url,word,search_country) values(?,?,?)";
	private static QueryRunner query = new QueryRunner(Db.getDs());
	private static String metaWord = "word";
	private static String metaId = "id";

	private static boolean mock = true;

	public static void main(String[] args) throws Exception {
		// String query = URLEncoder.encode("A B", "utf-8");
		// log.info("query:{}", query);
		// String baseUrl =
		// "http://www.alibaba.com/trade/search?fsb=y&IndexArea=company_en&CatId=&SearchText={0}";
		// String getLocationUrl = MessageFormat.format(baseUrl, query);
		// log.info("getLocationUrl:{}", getLocationUrl);
		String a = "Vacuum Transformer <strong>Oil</strong> Purifier,Vacuum Lubricating <strong>Oil</strong> Purifier,Vacuum Turbine <strong>Oil</strong> Purifier,Non-Vacuum <strong>Oil</strong> Purifier";
		System.out.println(a);
		a = a.replaceAll("<\\w{1,}>", "");
		a = a.replaceAll("</\\w{1,}>", "");
		System.out.println(a);
	}

	class ProxyHost {
		private String host;
		private Integer port;

		public String getHost() {
			return host;
		}

		public void setHost(String host) {
			this.host = host;
		}

		public Integer getPort() {
			return port;
		}

		public void setPort(Integer port) {
			this.port = port;
		}

	}

	private static Map<String, ProxyHost> proxyMp;
	private static long time = System.currentTimeMillis();

	// public ProxyHost getHost() {
	// if (proxyMp == null) {
	// synchronized (DemoSpider.class) {
	// if (proxyMp == null) {
	// proxyMp = new HashMap<String, DemoSpider.ProxyHost>();
	// InputStream in = ClassLoader.getSystemResourceAsStream("proxy");
	// BufferedReader reader = new BufferedReader(new InputStreamReader(in));
	// String line = null;
	// try {
	// while ((line = reader.readLine()) != null) {
	// ProxyHost host = new ProxyHost();
	// int sep = line.indexOf(":");
	// String ip = line.substring(0, sep);
	// String port = line.substring(sep + 1);
	// host.setHost(ip);
	// host.setPort(Integer.valueOf(port));
	// proxyMp.put(line, host);
	// }
	// } catch (IOException e) {
	// // TODO Auto-generated catch block
	// e.printStackTrace();
	// }
	// }
	// }
	// }
	//
	// return null;// proxyMp.values().;
	// }

	private static HttpGet getHttpGet(String url) {
		HttpGet httpGet = new HttpGet(url);
		httpGet.setHeader("User-Agent",
				"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36");
		httpGet.setHeader("Pragma", "no-cache");
		httpGet.setHeader("Referer", url);
		httpGet.setHeader("Cache-Control", "no-cache");
		httpGet.setConfig(config);
		return httpGet;

	}

	private static Map<String, Object> getKeywordMp() {
		Map<String, Object> keyWordMp = null;
		int count = 30;
		do {
			try {

				Connection conn = null;
				try {
					conn = Db.getDs().getConnection();
					conn.setAutoCommit(false);
					log.info("获取keyWordMp");
					keyWordMp = query.query(conn, queryUnuseWord, new MapHandler());
					if (keyWordMp != null && keyWordMp.get(metaWord) != null && StringUtils.isNotBlank(keyWordMp.get(metaWord).toString())) {
						log.info("更新keyword:{}的使用事件", keyWordMp.get(metaWord).toString());
						int rowNum = query.update(conn, updateUseWord, keyWordMp.get(metaId));
						log.info("更新keyword,affact num of row:{}", rowNum);
					}
					conn.commit();
				} catch (Exception e) {
					log.error("出错:", e);
					if (conn != null) {
						conn.rollback();
					}
				} finally {
					if (conn != null) {
						conn.setAutoCommit(true);
						conn.close();
					}
				}
			} catch (Exception e) {
				log.info("出错:", e);
			}
			count--;
			if (count <= 0) {
				break;
			}
		} while (keyWordMp == null || keyWordMp.get(metaWord) == null || StringUtils.isBlank(keyWordMp.get(metaWord).toString()));
		if (keyWordMp == null) {
			keyWordMp = new HashMap<String, Object>(0);
		}
		return keyWordMp;
	}

	private static String beautyKeyword(String keyword) {
		String[] wordSegs = keyword.split(" ");
		StringBuilder keywordBuilder = new StringBuilder();

		for (String w : wordSegs) {
			String trimW = w.trim();
			if (StringUtils.isBlank(trimW)) {
				continue;
			}
			if (keywordBuilder.length() == 0) {
				keywordBuilder.append(trimW);
			} else {
				keywordBuilder.append(" ").append(trimW);
			}
		}
		return keywordBuilder.toString();
	}

	private static List<String> getCountries(Document doc) {
		// J-filter-action-Location
		Elements locationDiv = doc.select("#J-filter-action-Location");
		log.info("Location{}", locationDiv.size());
		List<String> countryList = new ArrayList<String>();
		if (!locationDiv.isEmpty()) {
			// 提取country
			Elements counties = locationDiv.select("input[type=checkbox]");
			log.info("获取到countryElement:{}", counties.size());
			for (Element el : counties) {
				String countryData = el.attr("data-rev");
				log.info("国家:{}", countryData);
				countryList.add(countryData);
			}
		}
		return countryList;

	}

	private static StringBuilder getCountryBaseUrl(Document doc) {
		Elements locationDiv = doc.select("#J-filter-action-Location");
		Elements scriptTags = locationDiv.select("script");
		log.info("scripty:{}", scriptTags.size());
		StringBuilder countryBaseUrl = new StringBuilder();
		for (Element scriptTag : scriptTags) {
			String js = scriptTag.data();
			log.info("js:{}", js);
			int beginUrlIndex = js.indexOf("url:'");
			int endUrlIndex = js.indexOf("',", beginUrlIndex);
			String url = js.substring(beginUrlIndex + 5, endUrlIndex);
			log.info("url in js:{}", url);
			if (StringUtils.isNotBlank(url)) {
				countryBaseUrl.append(url);
				break;
			}
		}
		return countryBaseUrl;
	}

	public static void run() {

		CloseableHttpClient client = HttpClients.createDefault();

		boolean first = true;

		String baseUrl = "http://www.alibaba.com/trade/search?fsb=y&IndexArea=company_en&CatId=&SearchText={0}";

		String coutryUrl = "http://www.alibaba.com/trade/search?IndexArea=company_en&SearchText={0}&atm=&f0=y&Country={1}";

		while (true) {

			CloseableHttpResponse response = null;
			try {
				response = null;
				// ===========获取一个关键词============
				Map<String, Object> keyWordMp = getKeywordMp();

				// ===============获取第一个页面取得所有国家信息===============
				String word = keyWordMp.get("word").toString().trim();
				// 格式化关键字
				word = beautyKeyword(word);

				log.info("获取到keyword:{}", word);
				String encodeWord = URLEncoder.encode(word, "utf-8");

				String getLocationUrl = MessageFormat.format(baseUrl, encodeWord);
				log.info("getLocationUrl:{}", getLocationUrl);
				
//				Document doc = HttpHelper.get(getLocationUrl, 1);
				
				
				HttpGet httpGet = getHttpGet(getLocationUrl);

				log.info("开始http请求,获取location");
				response = client.execute(httpGet);

				if (response.getStatusLine().getStatusCode() == 200) {
					redirectNum = 0;
					log.info("获取location的entity");
					HttpEntity entity = response.getEntity();
					do {
						if (entity == null) {
							log.error("entity is null");
							break;
						}
						log.info("location的entity转换成document");
						Document doc = Jsoup.parse(EntityUtils.toString(entity));
						// 获取国家地区
						List<String> countryList = getCountries(doc);

						// ========按国家遍历========

						if (!countryList.isEmpty()) {
							// 获取按国家遍历的地址
							StringBuilder countryBaseUrl = getCountryBaseUrl(doc);
							// 按国家遍历
							for (String ct : countryList) {
								// 组装国家url
								String queryUrl = null;
								if (countryBaseUrl.length() == 0) {
									String keywordformat = encodeWord.replaceAll("+", "_");
									queryUrl = MessageFormat.format(coutryUrl, keywordformat, ct);
								} else {
									queryUrl = countryBaseUrl.toString() + "&Country=" + ct;
								}
								if (queryUrl == null) {
									log.info("queryUrl is null");
									break;
								}
								log.info("queryUrl:{}", queryUrl);
								Map<String, Object> meta = new HashMap<String, Object>();
								meta.put("country", ct);
								meta.put("word", word);
								recyclePage(queryUrl, meta, true);
							}

						} else {
							Map<String, Object> meta = Collections.emptyMap();
							recyclePage(getLocationUrl, meta, true);
						}

					} while (false);

					if (mock) {
						break;

					}

				} else if (response.getStatusLine().getStatusCode() == 403 || response.getStatusLine().getStatusCode() == 301
						|| response.getStatusLine().getStatusCode() == 302) {
					log.info("http请求location返回:{}", response.getStatusLine().getStatusCode());
					redirectNum++;
					log.info("redirectNum:{}", redirectNum);
					if (redirectNum > maxRedirect) {
						try {
							DateTime now = DateTime.now(TimeZone.getTimeZone("Asia/Shanghai"));
							log.info("返回{},sleep@time:{}", response.getStatusLine().getStatusCode(), now.format("YYYY-MM-DD hh:mm:ss"));
							Thread.sleep(1000 * 60 * 15);
						} catch (InterruptedException e) {
							// TODO Auto-generated catch block
							e.printStackTrace();
						}
						DateTime now = DateTime.now(TimeZone.getTimeZone("Asia/Shanghai"));
						log.info("返回{},wake@time:{}", response.getStatusLine().getStatusCode(), now.format("YYYY-MM-DD hh:mm:ss"));
					}
				}

			} catch (Exception e) {
				log.error("出错啦!!There is an error:", e);
			} finally {
				if (response != null) {
					log.info("close response");
					try {
						response.close();
					} catch (IOException e) {
						log.info("", e);
					}
				}
			}

		}
	}

	/**
	 * 获取分页信息
	 * 
	 * @param doc
	 * @return
	 */
	private static List<String> getPages(Document doc) {
		Elements naviBar = doc.select("#J-m-pagination");
		if (naviBar.isEmpty()) {
			log.info("naviBar is empty");
			return Collections.emptyList();
		}
		Elements navis = naviBar.select(".navi");
		if (navis.isEmpty()) {
			log.info("navis is empty");
			return Collections.emptyList();
		}
		Elements as = navis.select("a");
		if (as.isEmpty()) {
			log.info("a is empty");
			return Collections.emptyList();
		}
		List<String> pagesLst = new ArrayList<String>();
		try {
			Element firstA = as.first();
			log.info("page2:{}", firstA.toString());
			String beginHref = firstA.absUrl("href");
			log.info("beginPage:{}", beginHref);
			Element lastA = as.get(as.size() - 2);
			String endHref = lastA.absUrl("href");
			log.info("endPage:{}", endHref);
			do {
				int index = beginHref.lastIndexOf("/");
				if (index < 0) {
					break;
				}
				int lastDot = beginHref.lastIndexOf(".");
				if (lastDot < 0) {
					break;
				}
				String startV = beginHref.substring(index + 1, lastDot);
				log.info("startV:{}", startV);

				index = endHref.lastIndexOf("/");
				if (index < 0) {
					break;
				}
				lastDot = endHref.lastIndexOf(".");
				if (lastDot < 0) {
					break;
				}
				String endV = endHref.substring(index + 1, lastDot);
				log.info("endV:{}", endV);
				String baseUrl = beginHref.substring(0, index + 1);
				Integer startIntV = Integer.valueOf(startV);
				Integer endIntV = Integer.valueOf(endV);
				for (int i = startIntV.intValue(); i <= endIntV.intValue(); i++) {
					String url = baseUrl + i + ".html";
					pagesLst.add(url);
				}

			} while (false);

		} catch (Exception e) {
			log.info("", e);
		}

		return pagesLst;
	}

	/**
	 * 查询结果翻页继续查
	 * 
	 * @param url
	 */
	public static void recyclePage(String url, Map<String, Object> meta, boolean next) {
		CloseableHttpClient client = HttpClients.createDefault();
		CloseableHttpResponse response = null;// client.execute(httpGet);
		try {
			
			HttpGet httpGet = getHttpGet(url);
			log.info("request:{}", url);
			response = client.execute(httpGet);
			if (response.getStatusLine().getStatusCode() == 200) {
				redirectNum = 0;
				do {
					HttpEntity entity = response.getEntity();
					if (entity == null) {
						log.error("page entity is null");
						break;
					}
					log.info("page的entity转换成document");
					Document doc = Jsoup.parse(EntityUtils.toString(entity));
					collectData(doc, meta);
					if (next) {
						List<String> nextPageLst = getPages(doc);
						for (String nextUrl : nextPageLst) {
							log.info("下一页:{}", nextUrl);
							recyclePage(nextUrl, meta, false);
						}

					}
				} while (false);
			} else if (response.getStatusLine().getStatusCode() == 403 || response.getStatusLine().getStatusCode() == 301
					|| response.getStatusLine().getStatusCode() == 302) {
				log.info("http请求location返回:{}", response.getStatusLine().getStatusCode());
				redirectNum++;
				log.info("redirectNum:{}", redirectNum);
				if (redirectNum > maxRedirect) {
					try {
						DateTime now = DateTime.now(TimeZone.getTimeZone("Asia/Shanghai"));
						log.info("返回{},sleep@time:{}", response.getStatusLine().getStatusCode(), now.format("YYYY-MM-DD hh:mm:ss"));
						Thread.sleep(1000 * 60 * 15);
					} catch (InterruptedException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
					DateTime now = DateTime.now(TimeZone.getTimeZone("Asia/Shanghai"));
					log.info("返回{},wake@time:{}", response.getStatusLine().getStatusCode(), now.format("YYYY-MM-DD hh:mm:ss"));
				}

			}
		} catch (Exception e) {
			log.info("出错:", e);
		} finally {
			if (response != null) {
				try {
					response.close();
				} catch (IOException e) {
					log.info("出错:", e);
				}
			}
		}

	}

	private static void collectData(Document doc, Map<String, Object> meta) {
		Elements divs = doc.select(".f-icon.m-item ");
		log.info("f-icon m-item size:{}", divs.size());
		for (Element div : divs) {
			// 详情地址
			Elements company = div.select("div.company a.cd");
			if (company.size() != 1) {
				continue;
			}
			String href = company.first().absUrl("href");
			if (StringUtils.isBlank(href)) {
				log.info("company detail href is blank");
				continue;
			}
			if (!href.startsWith("http")) {
				log.info("company detail href is not start with http");
				break;
			}
			log.info("company detail href is:{}", href);
			try {
				Object word = meta.get("word");
				if (word == null) {
					word = "";
				}
				Object country = meta.get("country");
				if (country == null) {
					country = "";
				}
				Number id = query.query("select id from record where url = ?", new ScalarHandler<Number>("id"), href);
				if (id == null) {
					log.info("insert word:{}, country:{}", word, country);
					query.update(insertUrl, href, word, country);
				} else {
					log.info("已存在:{}", href);
					continue;
				}

			} catch (Exception e) {
				log.info("出错了:", e);
				continue;
			}

			// 获取金牌卖家年份
			do {
				Elements goldYears = div.select(".ico-year");
				if (goldYears.size() != 1) {
					break;
				}
				Element goldYear = goldYears.first();
				Elements span = goldYear.select("span");
				if (span.size() != 1) {
					break;
				}
				String className = span.attr("class");
				if (StringUtils.isBlank(className)) {
					break;
				}
				int gsIndex = className.indexOf("gs");
				String gsLevel = className.substring(gsIndex + "gs".length());
				try {
					int level = Integer.valueOf(gsLevel).intValue();
					log.info("更新goldlevel:{}, url:{}", level, href);
					query.update("update record set gold_supplier = ? where url=?", level, href);

				} catch (Exception e) {
					log.info("出错:", e);
				}

			} while (false);

			do {
				// ico ico-ta
				Elements tas = div.select(".ico-ta");
				if (tas.size() != 1) {
					break;
				}

				try {
					log.info("更新trade_assurance url:{}", href);
					query.update("update record set trade_assurance = 1 where url=?", href);

				} catch (Exception e) {
					log.info("出错:", e);
				}
			} while (false);

			do {
				try {
					Elements asdiv = div.select(".as.J-as");
					if (asdiv.size() != 1) {
						log.info(".as.J-as != 1");
						break;
					}
					Elements as = asdiv.select("a");
					for (Element a : as) {
						String aHref = a.absUrl("href");
						if (StringUtils.isBlank(aHref)) {
							continue;
						}
						if (aHref.indexOf("capability_assessment") >= 0) {
							// Assessed Supplier
							query.update("update record set assessed_supplier=1 where url=?", href);
							break;
						}
					}

				} catch (Exception e) {
					log.info("出错:", e);
				}

			} while (false);

			do {
				try {
					Elements rights = div.select(".right");
					if (rights.size() != 1) {
						log.info(".right size != 1");
						break;
					}
					Elements attrLst = rights.select(".attrs .attr");
					for (Element attr : attrLst) {
						String name = attr.select(".name").html();
						if ("Main Products:".equalsIgnoreCase(name)) {
							String v = attr.select(".value").html();
							if (StringUtils.isBlank(v)) {
								continue;
							}
							v = v.replaceAll("<\\w{1,}>", "");
							v = v.replaceAll("</\\w{1,}>", "");
							log.info("保存product:{}, url:{}", v, href);
							query.update("update record set products=? where url=?", v, href);
						} else if ("Country/Region:".equalsIgnoreCase(name)) {
							String v = attr.select(".value").first().select("span").first().html();
							if (StringUtils.isBlank(v)) {
								continue;
							}
							log.info("保存region:{}, url:{}", v, href);
							query.update("update record set region=?  where url=?", v, href);
						} else if ("Total Revenue:".equalsIgnoreCase(name)) {
							String v = attr.select(".value").first().select("span").first().html();
							if (StringUtils.isBlank(v)) {
								continue;
							}
							log.info("保存total revenue:{}, url:{}", v, href);
							query.update("update record set total_revenue=?  where url=?", v, href);
						} else if ("Top 3 Markets:".equalsIgnoreCase(name)) {
							Elements spans = attr.select(".value").first().select("span");
							if (spans.size() == 0) {
								continue;
							}
							StringBuilder v = new StringBuilder();
							for (Element span : spans) {
								v.append(span.html()).append(",");
							}
							log.info("保存top3markets:{}, url:{}", v.toString(), href);
							query.update("update record set top3_markets=?  where url=?", v.toString(), href);
						} else {
							Elements resp = attr.select("a.sts");
							if (resp.size() != 1) {
								log.info("resp.size != 1");
								continue;
							}
							String percent = resp.first().html();
							if (StringUtils.isBlank(percent)) {
								log.info("precent is blank");
								continue;
							}
							int i = percent.indexOf("%");
							if (i < 0) {
								log.info("not find %:{}", percent);
								continue;
							}
							String v = percent.substring(0, i);
							double percentV = Double.valueOf(v);
							percentV = percentV * 100;
							log.info("response rate:{}", percentV);
							query.update("update record set resp_rate=? where url=?", Double.valueOf(percentV).intValue(), href);

						}
					}

				} catch (Exception e) {
					log.info("出错:", e);
				}
			} while (false);

			detailDataCollect(href);

		}
	}

	private static void detailDataCollect(String url) {

		CloseableHttpClient client = HttpClients.createDefault();
		CloseableHttpResponse response = null;// client.execute(httpGet);
		try {
			if (!first) {
				log.info("sleep a while");
				Integer seconds = Integer.valueOf(3);
				Integer delay = random.nextInt(seconds);
				delay = (seconds + delay) * 1000;
				log.info("sleep time:{}", delay);
				Thread.sleep(delay);
				log.info("wake after rest a while");
				first = false;
			} else {
				first = false;
			}
			HttpGet httpGet = getHttpGet(url);
			log.info("request:{}", url);
			response = client.execute(httpGet);
			if (response.getStatusLine().getStatusCode() == 200) {
				redirectNum = 0;
				do {
					HttpEntity entity = response.getEntity();
					if (entity == null) {
						log.error("page entity is null");
						break;
					}
					log.info("page的entity转换成document");
					Document doc = Jsoup.parse(EntityUtils.toString(entity));
					collectDetailData(url, doc);
				} while (false);
			} else if (response.getStatusLine().getStatusCode() == 403 || response.getStatusLine().getStatusCode() == 301
					|| response.getStatusLine().getStatusCode() == 302) {
				log.info("http请求location返回:{}", response.getStatusLine().getStatusCode());
				redirectNum++;
				log.info("redirectNum:{}", redirectNum);
				if (redirectNum > maxRedirect) {
					try {
						DateTime now = DateTime.now(TimeZone.getTimeZone("Asia/Shanghai"));
						log.info("返回{},sleep@time:{}", response.getStatusLine().getStatusCode(), now.format("YYYY-MM-DD hh:mm:ss"));
						Thread.sleep(1000 * 60 * 15);
					} catch (InterruptedException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
					DateTime now = DateTime.now(TimeZone.getTimeZone("Asia/Shanghai"));
					log.info("返回{},wake@time:{}", response.getStatusLine().getStatusCode(), now.format("YYYY-MM-DD hh:mm:ss"));
				}

			}
		} catch (Exception e) {
			log.info("出错:", e);
		} finally {
			if (response != null) {
				try {
					response.close();
				} catch (IOException e) {
					log.info("出错:", e);
				}
			}
		}
	}

	private static void collectDetailData(String url, Document doc) {

		Element exper = doc.select(".info-item.experience-info div").first();
		if (exper != null) {
			Element div = exper.select("div").first();
			if (div != null) {
				String year = div.html();
				if (StringUtils.isNotBlank(year)) {
					int i = year.indexOf("Established");
					if (i >= 0) {
						String y = year.substring(i + "Established".length());
						if (y != null) {
							y = y.trim();
							try {
								query.update("update record set experience=? where url=?", Integer.valueOf(y), url);

							} catch (Exception e) {
								log.info("出错:", e);
							}
						}
					}
				}
			}
		}

		// #contact-person
		Element person = doc.select("#contact-person").first();
		if (person == null) {
			log.info("contact-person is null");
			return;
		}
		do {
			try {
				Element info = person.select(".contact-overview .contact-info").first();
				if (info == null) {
					break;
				}
				Element name = info.select(".name").first();
				if (name != null) {
					String strName = name.html();
					if (StringUtils.isNotBlank(strName)) {
						strName = strName.trim();
						query.update("update record set name=? where url=?", strName, url);
					}
				}
				Element dl = info.select(".dl-horizontal").first();
				if (dl != null) {
					String dtName = null;
					for (Element e : dl.children()) {
						if ("dt".equalsIgnoreCase(e.tagName())) {
							dtName = e.html();
						}
						if ("dd".equalsIgnoreCase(e.tagName())) {
							if (StringUtils.isNotBlank(dtName)) {
								if ("Department:".equalsIgnoreCase(dtName)) {
									query.update("update record set department=? where url=?", e.html(), url);
								} else if ("Job Title:".equalsIgnoreCase(dtName)) {
									query.update("update record set job_title=? where url=?", e.html(), url);
								}
							}
						}
					}
				}
				// =======================
				Element detailDl = person.select(".contact-detail dl.dl-horizontal").first();
				if (detailDl != null) {
					String dtName = null;
					for (Element e : detailDl.children()) {
						if ("dt".equalsIgnoreCase(e.tagName())) {
							dtName = e.html();
						}
						if ("dd".equalsIgnoreCase(e.tagName())) {
							if (StringUtils.isNotBlank(dtName)) {
								if ("Telephone:".equalsIgnoreCase(dtName)) {
									query.update("update record set tel=? where url=?", e.html(), url);
								} else if ("Mobile Phone:".equalsIgnoreCase(dtName)) {
									query.update("update record set mobile=? where url=?", e.html(), url);
								} else if ("Fax:".equalsIgnoreCase(dtName)) {
									query.update("update record set fax=? where url=?", e.html(), url);
								} else if ("Address:".equalsIgnoreCase(dtName)) {
									query.update("update record set address=? where url=?", e.html(), url);
								} else if ("Zip:".equalsIgnoreCase(dtName)) {
									query.update("update record set zip=? where url=?", e.html(), url);
								} else if ("Country/Region:".equalsIgnoreCase(dtName)) {
									query.update("update record set region=? where url=?", e.html(), url);
								} else if ("Province/State:".equalsIgnoreCase(dtName)) {
									query.update("update record set province=? where url=?", e.html(), url);
								} else if ("City:".equalsIgnoreCase(dtName)) {
									query.update("update record set city=? where url=?", e.html(), url);
								}
							}
						}
					}
				}

			} catch (Exception e) {
				log.info("出错:", e);
			}

		} while (false);

		do {
			// company-contact-information

			Element companyInfo = doc.select(".company-contact-information").first();
			if (companyInfo == null) {
				log.info("company-contact-information is null");
				break;
			}
			Element tbody = companyInfo.select(".company-info-data.table").first();
			if (tbody == null) {
				log.info("company-info-data table is null");
				break;
			}
			Elements trs = tbody.select("tr");
			for (Element tr : trs) {
				if (tr.hasClass("hide")) {
					continue;
				}
				Element th = tr.select("th").first();
				if (th == null) {
					continue;
				}
				String thstr = th.html();
				try {

					if ("Company Name:".equalsIgnoreCase(thstr)) {
						Element td = th.nextElementSibling();
						if (td == null) {
							continue;
						}
						query.update("update record set company_name=? where url=?", td.html(), url);
					} else if ("Operational Address:".equalsIgnoreCase(thstr)) {
						Element td = th.nextElementSibling();
						if (td == null) {
							continue;
						}
						query.update("update record set operational_address=? where url=?", td.html(), url);
					} else if ("Website:".equalsIgnoreCase(thstr)) {
						Element td = th.nextElementSibling();
						if (td == null) {
							continue;
						}
						Elements as = td.select("a");
						StringBuilder sb = new StringBuilder();
						for (Element a : as) {
							sb.append(a.absUrl("href")).append(",");
						}
						if (sb.length() == 0) {
							continue;
						}
						query.update("update record set website=? where url=?", sb.toString(), url);
					} else if ("Website on alibaba.com:".equalsIgnoreCase(thstr)) {
						Element td = th.nextElementSibling();
						if (td == null) {
							continue;
						}
						Elements as = td.select("a");
						StringBuilder sb = new StringBuilder();
						for (Element a : as) {
							sb.append(a.absUrl("href")).append(",");
						}
						if (sb.length() == 0) {
							continue;
						}
						query.update("update record set website_on_alibaba=? where url=?", sb.toString(), url);
					}
				} catch (Exception e) {
					log.info("出错:", e);
				}
			}

		} while (false);
	}

}
